# encoding: utf-8

import requests
import bs4

start_url = ["https://search.51job.com/list/090200,000000,0000,00,9,99,Python%2B%25E5%25BC%2580%25E5%258F%2591,2,2.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="]

sess = requests.Session()


def spider_url(sess, url):
    print("Spidering " + url)
    response = sess.get(url)
    response.encoding = "gbk"
    response.raise_for_status()
    bs = bs4.BeautifulSoup(response.text, features="lxml")
    jobs_list = bs.find("div", attrs={"class": "dw_table", "id": "resultList"})
    if not jobs_list:
        raise Exception("Can not find the tag")

    for item in jobs_list.find_all("div", attrs={"class": "el"}):
        try:
            title_tag = item.p.span.a.attrs["title"]
            details = list(item.children)
            company = details[1].a.attrs["title"]
            address = details[5].text
            salary_range = details[7].text
            yield {
                "job_title": title_tag,
                "company": company,
                "address": address,
                "salary": salary_range
            }
        except Exception as e:
            # traceback.print_exc()
            # print(str(e))
            pass
            # print(item)

    pages_url = bs.find("div", attrs={"class": "p_in"})
    for next_page_url in pages_url.find_all("li", attrs=None):
        try:
            if not next_page_url.attrs:
                if next_page_url.a.attrs["href"] not in start_url:
                    start_url.append(next_page_url.a.attrs["href"])
        except Exception as e:
            pass

# for url in start_url:
#     items = spider_url(sess, url)
#     for item in items:
#         print(item)
#     # print(item)
# sess.close()
# do_url(session, start_url)
# session.close()
# print(dir(response))


response = requests.get("http://www.xiaohuar.com/d/file/20190526/small4f6fbdcfdf5f37e1c2cb3875b050399c1558854131.jpg")
with open("./xioahua.jpg", "wb") as fp:
    fp.write(response.content)